{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input\"))\n",
    "\n",
    "import time\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from tqdm import tqdm\n",
    "import math\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import GridSearchCV, StratifiedKFold, KFold\n",
    "from sklearn.metrics import f1_score, roc_auc_score\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\n",
    "from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\n",
    "from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate\n",
    "from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\n",
    "from keras.optimizers import Adam\n",
    "from keras.models import Model\n",
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers\n",
    "from keras.layers import concatenate\n",
    "from keras.callbacks import *\n",
    "\n",
    "\n",
    "def load_and_prec(prefix):\n",
    "    train_df = pd.read_pickle(\"../pickle/{}_text_train.pickle\".format(prefix))\n",
    "    test_df = pd.read_pickle(\"../pickle/{}_text_test.pickle\".format(prefix))\n",
    "    print(\"Train shape : \",train_df.shape)\n",
    "    print(\"Test shape : \",test_df.shape)\n",
    "    \n",
    "    ## fill up the missing values\n",
    "    if prefix=='active':\n",
    "        train_X = train_df[\"appid\"].map(lambda x:x.split('#')).fillna(\"_##_\").values\n",
    "        test_X = test_df[\"appid\"].map(lambda x:x.split('#')).fillna(\"_##_\").values\n",
    "    else:\n",
    "        train_X = train_df[\"appid\"].fillna(\"_##_\").values\n",
    "        test_X = test_df[\"appid\"].fillna(\"_##_\").values\n",
    "\n",
    "    ## Tokenize the sentences\n",
    "    tokenizer = Tokenizer(num_words=max_features)\n",
    "    tokenizer.fit_on_texts(list(train_X))\n",
    "    train_X = tokenizer.texts_to_sequences(train_X)\n",
    "    test_X = tokenizer.texts_to_sequences(test_X)\n",
    "\n",
    "    ## Pad the sentences \n",
    "    train_X = pad_sequences(train_X, maxlen=maxlen)\n",
    "    test_X = pad_sequences(test_X, maxlen=maxlen)\n",
    "\n",
    "    ## Get the target values\n",
    "    train_y = train_df['age_group'].values\n",
    "    \n",
    "    #shuffling the data\n",
    "    np.random.seed(2019)\n",
    "    trn_idx = np.random.permutation(len(train_X))\n",
    "\n",
    "    train_X = train_X[trn_idx]\n",
    "    train_y = train_y[trn_idx]\n",
    "    \n",
    "    return train_X, test_X, train_y, tokenizer.word_index\n",
    "\n",
    "from glove import *\n",
    "def load_glove(word_index,max_features,prefix):\n",
    "    \n",
    "    EMBEDDING_FILE = '../vector/{}_glove300.model'.format(prefix)\n",
    "    k = Glove.load(EMBEDDING_FILE)\n",
    "    embeddings_index = []\n",
    "    for i in tqdm(k.dictionary.keys()):\n",
    "        embeddings_index.append((i,k.word_vectors[k.dictionary[i]]))\n",
    "\n",
    "    embeddings_index = dict(pd.DataFrame(embeddings_index).values)\n",
    "    all_embs = np.stack(embeddings_index.values())\n",
    "    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n",
    "    embed_size = all_embs.shape[1]\n",
    "\n",
    "    # word_index = tokenizer.word_index\n",
    "    nb_words = min(max_features, len(word_index))\n",
    "    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n",
    "    for word, i in word_index.items():\n",
    "        if i >= max_features: continue\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n",
    "            \n",
    "    return embedding_matrix \n",
    "    \n",
    "def load_fasttext(word_index,max_features):    \n",
    "    EMBEDDING_FILE = '../vector/fasttext300.model'\n",
    "    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n",
    "    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n",
    "\n",
    "    all_embs = np.stack(embeddings_index.values())\n",
    "    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n",
    "    embed_size = all_embs.shape[1]\n",
    "\n",
    "    # word_index = tokenizer.word_index\n",
    "    nb_words = min(max_features, len(word_index))\n",
    "    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n",
    "    for word, i in word_index.items():\n",
    "        if i >= max_features: continue\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n",
    "\n",
    "    return embedding_matrix\n",
    "\n",
    "def load_w2v(word_index,max_features,prefix):    \n",
    "    EMBEDDING_FILE = '../vector/{}_w2v300.model'.format(prefix)\n",
    "    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n",
    "    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n",
    "\n",
    "    all_embs = np.stack(embeddings_index.values())\n",
    "    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n",
    "    embed_size = all_embs.shape[1]\n",
    "\n",
    "    # word_index = tokenizer.word_index\n",
    "    nb_words = min(max_features, len(word_index))\n",
    "    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n",
    "    for word, i in word_index.items():\n",
    "        if i >= max_features: continue\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n",
    "\n",
    "    return embedding_matrix\n",
    "\n",
    "class Attention(Layer):\n",
    "    def __init__(self, step_dim,\n",
    "                 W_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "        self.supports_masking = True\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.step_dim = step_dim\n",
    "        self.features_dim = 0\n",
    "        super(Attention, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        self.features_dim = input_shape[-1]\n",
    "\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        features_dim = self.features_dim\n",
    "        step_dim = self.step_dim\n",
    "\n",
    "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n",
    "                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
    "\n",
    "        if self.bias:\n",
    "            eij += self.b\n",
    "\n",
    "        eij = K.tanh(eij)\n",
    "\n",
    "        a = K.exp(eij)\n",
    "\n",
    "        if mask is not None:\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return input_shape[0],  self.features_dim\n",
    "    \n",
    "class CyclicLR(Callback):\n",
    "    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n",
    "    The method cycles the learning rate between two boundaries with\n",
    "    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).\n",
    "    The amplitude of the cycle can be scaled on a per-iteration or \n",
    "    per-cycle basis.\n",
    "    This class has three built-in policies, as put forth in the paper.\n",
    "    \"triangular\":\n",
    "        A basic triangular cycle w/ no amplitude scaling.\n",
    "    \"triangular2\":\n",
    "        A basic triangular cycle that scales initial amplitude by half each cycle.\n",
    "    \"exp_range\":\n",
    "        A cycle that scales initial amplitude by gamma**(cycle iterations) at each \n",
    "        cycle iteration.\n",
    "    For more detail, please see paper.\n",
    "    \n",
    "    # Example\n",
    "        ```python\n",
    "            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n",
    "                                step_size=2000., mode='triangular')\n",
    "            model.fit(X_train, Y_train, callbacks=[clr])\n",
    "        ```\n",
    "    \n",
    "    Class also supports custom scaling functions:\n",
    "        ```python\n",
    "            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n",
    "            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n",
    "                                step_size=2000., scale_fn=clr_fn,\n",
    "                                scale_mode='cycle')\n",
    "            model.fit(X_train, Y_train, callbacks=[clr])\n",
    "        ```    \n",
    "    # Arguments\n",
    "        base_lr: initial learning rate which is the\n",
    "            lower boundary in the cycle.\n",
    "        max_lr: upper boundary in the cycle. Functionally,\n",
    "            it defines the cycle amplitude (max_lr - base_lr).\n",
    "            The lr at any cycle is the sum of base_lr\n",
    "            and some scaling of the amplitude; therefore \n",
    "            max_lr may not actually be reached depending on\n",
    "            scaling function.\n",
    "        step_size: number of training iterations per\n",
    "            half cycle. Authors suggest setting step_size\n",
    "            2-8 x training iterations in epoch.\n",
    "        mode: one of {triangular, triangular2, exp_range}.\n",
    "            Default 'triangular'.\n",
    "            Values correspond to policies detailed above.\n",
    "            If scale_fn is not None, this argument is ignored.\n",
    "        gamma: constant in 'exp_range' scaling function:\n",
    "            gamma**(cycle iterations)\n",
    "        scale_fn: Custom scaling policy defined by a single\n",
    "            argument lambda function, where \n",
    "            0 <= scale_fn(x) <= 1 for all x >= 0.\n",
    "            mode paramater is ignored \n",
    "        scale_mode: {'cycle', 'iterations'}.\n",
    "            Defines whether scale_fn is evaluated on \n",
    "            cycle number or cycle iterations (training\n",
    "            iterations since start of cycle). Default is 'cycle'.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n",
    "                 gamma=1., scale_fn=None, scale_mode='cycle'):\n",
    "        super(CyclicLR, self).__init__()\n",
    "\n",
    "        self.base_lr = base_lr\n",
    "        self.max_lr = max_lr\n",
    "        self.step_size = step_size\n",
    "        self.mode = mode\n",
    "        self.gamma = gamma\n",
    "        if scale_fn == None:\n",
    "            if self.mode == 'triangular':\n",
    "                self.scale_fn = lambda x: 1.\n",
    "                self.scale_mode = 'cycle'\n",
    "            elif self.mode == 'triangular2':\n",
    "                self.scale_fn = lambda x: 1/(2.**(x-1))\n",
    "                self.scale_mode = 'cycle'\n",
    "            elif self.mode == 'exp_range':\n",
    "                self.scale_fn = lambda x: gamma**(x)\n",
    "                self.scale_mode = 'iterations'\n",
    "        else:\n",
    "            self.scale_fn = scale_fn\n",
    "            self.scale_mode = scale_mode\n",
    "        self.clr_iterations = 0.\n",
    "        self.trn_iterations = 0.\n",
    "        self.history = {}\n",
    "\n",
    "        self._reset()\n",
    "\n",
    "    def _reset(self, new_base_lr=None, new_max_lr=None,\n",
    "               new_step_size=None):\n",
    "        \"\"\"Resets cycle iterations.\n",
    "        Optional boundary/step size adjustment.\n",
    "        \"\"\"\n",
    "        if new_base_lr != None:\n",
    "            self.base_lr = new_base_lr\n",
    "        if new_max_lr != None:\n",
    "            self.max_lr = new_max_lr\n",
    "        if new_step_size != None:\n",
    "            self.step_size = new_step_size\n",
    "        self.clr_iterations = 0.\n",
    "        \n",
    "    def clr(self):\n",
    "        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n",
    "        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n",
    "        if self.scale_mode == 'cycle':\n",
    "            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n",
    "        else:\n",
    "            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)\n",
    "        \n",
    "    def on_train_begin(self, logs={}):\n",
    "        logs = logs or {}\n",
    "\n",
    "        if self.clr_iterations == 0:\n",
    "            K.set_value(self.model.optimizer.lr, self.base_lr)\n",
    "        else:\n",
    "            K.set_value(self.model.optimizer.lr, self.clr())        \n",
    "            \n",
    "    def on_batch_end(self, epoch, logs=None):\n",
    "        \n",
    "        logs = logs or {}\n",
    "        self.trn_iterations += 1\n",
    "        self.clr_iterations += 1\n",
    "\n",
    "        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n",
    "        self.history.setdefault('iterations', []).append(self.trn_iterations)\n",
    "\n",
    "        for k, v in logs.items():\n",
    "            self.history.setdefault(k, []).append(v)\n",
    "        \n",
    "        K.set_value(self.model.optimizer.lr, self.clr())\n",
    "        \n",
    "from keras.optimizers import *\n",
    "class RAdam(Optimizer):\n",
    "\n",
    "    \"\"\"RAdam optimizer.\n",
    "\n",
    "    # Arguments\n",
    "        lr: float >= 0. Learning rate.\n",
    "        beta_1: float, 0 < beta < 1. Generally close to 1.\n",
    "        beta_2: float, 0 < beta < 1. Generally close to 1.\n",
    "        epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`.\n",
    "        decay: float >= 0. Learning rate decay over each update.\n",
    "        weight_decay: float >= 0. Weight decay for each param.\n",
    "        amsgrad: boolean. Whether to apply the AMSGrad variant of this\n",
    "            algorithm from the paper \"On the Convergence of Adam and\n",
    "            Beyond\".\n",
    "        total_steps: int >= 0. Total number of training steps. Enable warmup by setting a positive value.\n",
    "        warmup_proportion: 0 < warmup_proportion < 1. The proportion of increasing steps.\n",
    "        min_lr: float >= 0. Minimum learning rate after warmup.\n",
    "    # References\n",
    "        - [Adam - A Method for Stochastic Optimization](https://arxiv.org/abs/1412.6980v8)\n",
    "        - [On the Convergence of Adam and Beyond](https://openreview.net/forum?id=ryQu7f-RZ)\n",
    "        - [On The Variance Of The Adaptive Learning Rate And Beyond](https://arxiv.org/pdf/1908.03265v1.pdf)\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n",
    "                 epsilon=None, decay=0., weight_decay=0., amsgrad=False,\n",
    "                 total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):\n",
    "        super(RAdam, self).__init__(**kwargs)\n",
    "        with K.name_scope(self.__class__.__name__):\n",
    "            self.iterations = K.variable(0, dtype='int64', name='iterations')\n",
    "            self.lr = K.variable(lr, name='lr')\n",
    "            self.beta_1 = K.variable(beta_1, name='beta_1')\n",
    "            self.beta_2 = K.variable(beta_2, name='beta_2')\n",
    "            self.decay = K.variable(decay, name='decay')\n",
    "            self.weight_decay = K.variable(weight_decay, name='weight_decay')\n",
    "            self.total_steps = K.variable(total_steps, name='total_steps')\n",
    "            self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')\n",
    "            self.min_lr = K.variable(min_lr, name='min_lr')\n",
    "        if epsilon is None:\n",
    "            epsilon = K.epsilon()\n",
    "        self.epsilon = epsilon\n",
    "        self.initial_decay = decay\n",
    "        self.initial_weight_decay = weight_decay\n",
    "        self.initial_total_steps = total_steps\n",
    "        self.amsgrad = amsgrad\n",
    "\n",
    "    def get_updates(self, loss, params):\n",
    "        grads = self.get_gradients(loss, params)\n",
    "        self.updates = [K.update_add(self.iterations, 1)]\n",
    "\n",
    "        lr = self.lr\n",
    "\n",
    "        if self.initial_decay > 0:\n",
    "            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, K.dtype(self.decay))))\n",
    "\n",
    "        t = K.cast(self.iterations, K.floatx()) + 1\n",
    "\n",
    "        if self.initial_total_steps > 0:\n",
    "            warmup_steps = self.total_steps * self.warmup_proportion\n",
    "            decay_steps = self.total_steps - warmup_steps\n",
    "            lr = K.switch(\n",
    "                t <= warmup_steps,\n",
    "                lr * (t / warmup_steps),\n",
    "                lr * (1.0 - K.minimum(t, decay_steps) / decay_steps),\n",
    "            )\n",
    "\n",
    "        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for (i, p) in enumerate(params)]\n",
    "        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for (i, p) in enumerate(params)]\n",
    "\n",
    "        if self.amsgrad:\n",
    "            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for (i, p) in enumerate(params)]\n",
    "        else:\n",
    "            vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params))]\n",
    "\n",
    "        self.weights = [self.iterations] + ms + vs + vhats\n",
    "\n",
    "        beta_1_t = K.pow(self.beta_1, t)\n",
    "        beta_2_t = K.pow(self.beta_2, t)\n",
    "\n",
    "        sma_inf = 2.0 / (1.0 - self.beta_2) - 1.0\n",
    "        sma_t = sma_inf - 2.0 * t * beta_2_t / (1.0 - beta_2_t)\n",
    "\n",
    "        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):\n",
    "            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g\n",
    "            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)\n",
    "\n",
    "            m_corr_t = m_t / (1.0 - beta_1_t)\n",
    "            if self.amsgrad:\n",
    "                vhat_t = K.maximum(vhat, v_t)\n",
    "                v_corr_t = K.sqrt(vhat_t / (1.0 - beta_2_t) + self.epsilon)\n",
    "                self.updates.append(K.update(vhat, vhat_t))\n",
    "            else:\n",
    "                v_corr_t = K.sqrt(v_t / (1.0 - beta_2_t) + self.epsilon)\n",
    "\n",
    "            r_t = K.sqrt((sma_t - 4.0) / (sma_inf - 4.0) *\n",
    "                         (sma_t - 2.0) / (sma_inf - 2.0) *\n",
    "                         sma_inf / sma_t)\n",
    "\n",
    "            p_t = K.switch(sma_t > 5, r_t * m_corr_t / v_corr_t, m_corr_t)\n",
    "\n",
    "            if self.initial_weight_decay > 0:\n",
    "                p_t += self.weight_decay * p\n",
    "\n",
    "            p_t = p - lr * p_t\n",
    "\n",
    "            self.updates.append(K.update(m, m_t))\n",
    "            self.updates.append(K.update(v, v_t))\n",
    "            new_p = p_t\n",
    "\n",
    "            # Apply constraints.\n",
    "            if getattr(p, 'constraint', None) is not None:\n",
    "                new_p = p.constraint(new_p)\n",
    "\n",
    "            self.updates.append(K.update(p, new_p))\n",
    "        return self.updates\n",
    "\n",
    "    def get_config(self):\n",
    "        config = {\n",
    "            'lr': float(K.get_value(self.lr)),\n",
    "            'beta_1': float(K.get_value(self.beta_1)),\n",
    "            'beta_2': float(K.get_value(self.beta_2)),\n",
    "            'decay': float(K.get_value(self.decay)),\n",
    "            'weight_decay': float(K.get_value(self.weight_decay)),\n",
    "            'epsilon': self.epsilon,\n",
    "            'amsgrad': self.amsgrad,\n",
    "            'total_steps': float(K.get_value(self.total_steps)),\n",
    "            'warmup_proportion': float(K.get_value(self.warmup_proportion)),\n",
    "            'min_lr': float(K.get_value(self.min_lr)),\n",
    "        }\n",
    "        base_config = super(RAdam, self).get_config()\n",
    "        return dict(list(base_config.items()) + list(config.items()))\n",
    "\n",
    "class Lookahead(object):\n",
    "    \"\"\"Add the [Lookahead Optimizer](https://arxiv.org/abs/1907.08610) functionality for [keras](https://keras.io/).\n",
    "\n",
    "    \"\"\"\n",
    "    def __init__(self, k=5, alpha=0.5):\n",
    "        self.k = k\n",
    "        self.alpha = alpha\n",
    "        self.count = 0\n",
    "\n",
    "    def inject(self, model):\n",
    "        \"\"\"Inject the Lookahead algorithm for the given model.\n",
    "        The following code is modified from keras's _make_train_function method.\n",
    "        See: https://github.com/keras-team/keras/blob/master/keras/engine/training.py#L497\n",
    "        \"\"\"\n",
    "        if not hasattr(model, 'train_function'):\n",
    "            raise RuntimeError('You must compile your model before using it.')\n",
    "\n",
    "        model._check_trainable_weights_consistency()\n",
    "\n",
    "        if model.train_function is None:\n",
    "            inputs = (model._feed_inputs +\n",
    "                      model._feed_targets +\n",
    "                      model._feed_sample_weights)\n",
    "            if model._uses_dynamic_learning_phase():\n",
    "                inputs += [K.learning_phase()]\n",
    "            fast_params = model._collected_trainable_weights\n",
    "\n",
    "            with K.name_scope('training'):\n",
    "                with K.name_scope(model.optimizer.__class__.__name__):\n",
    "                    training_updates = model.optimizer.get_updates(\n",
    "                        params=fast_params,\n",
    "                        loss=model.total_loss)\n",
    "                    slow_params = [K.variable(p) for p in fast_params]\n",
    "                fast_updates = (model.updates +\n",
    "                                training_updates +\n",
    "                                model.metrics_updates)\n",
    "\n",
    "                slow_updates, copy_updates = [], []\n",
    "                for p, q in zip(fast_params, slow_params):\n",
    "                    slow_updates.append(K.update(q, q + self.alpha * (p - q)))\n",
    "                    copy_updates.append(K.update(p, q))\n",
    "\n",
    "                # Gets loss and metrics. Updates weights at each call.\n",
    "                fast_train_function = K.function(\n",
    "                    inputs,\n",
    "                    [model.total_loss] + model.metrics_tensors,\n",
    "                    updates=fast_updates,\n",
    "                    name='fast_train_function',\n",
    "                    **model._function_kwargs)\n",
    "\n",
    "                def F(inputs):\n",
    "                    self.count += 1\n",
    "                    R = fast_train_function(inputs)\n",
    "                    if self.count % self.k == 0:\n",
    "                        K.batch_get_value(slow_updates)\n",
    "                        K.batch_get_value(copy_updates)\n",
    "                    return R\n",
    "                \n",
    "                model.train_function = F\n",
    "                \n",
    "class Capsule(Layer):\n",
    "    def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,\n",
    "                 activation='default', **kwargs):\n",
    "        super(Capsule, self).__init__(**kwargs)\n",
    "        self.num_capsule = num_capsule\n",
    "        self.dim_capsule = dim_capsule\n",
    "        self.routings = routings\n",
    "        self.kernel_size = kernel_size\n",
    "        self.share_weights = share_weights\n",
    "        if activation == 'default':\n",
    "            self.activation = squash\n",
    "        else:\n",
    "            self.activation = Activation(activation)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        super(Capsule, self).build(input_shape)\n",
    "        input_dim_capsule = input_shape[-1]\n",
    "        if self.share_weights:\n",
    "            self.W = self.add_weight(name='capsule_kernel',\n",
    "                                     shape=(1, input_dim_capsule,\n",
    "                                            self.num_capsule * self.dim_capsule),\n",
    "                                     # shape=self.kernel_size,\n",
    "                                     initializer='glorot_uniform',\n",
    "                                     trainable=True)\n",
    "        else:\n",
    "            input_num_capsule = input_shape[-2]\n",
    "            self.W = self.add_weight(name='capsule_kernel',\n",
    "                                     shape=(input_num_capsule,\n",
    "                                            input_dim_capsule,\n",
    "                                            self.num_capsule * self.dim_capsule),\n",
    "                                     initializer='glorot_uniform',\n",
    "                                     trainable=True)\n",
    "\n",
    "    def call(self, u_vecs):\n",
    "        if self.share_weights:\n",
    "            u_hat_vecs = K.conv1d(u_vecs, self.W)\n",
    "        else:\n",
    "            u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])\n",
    "\n",
    "        batch_size = K.shape(u_vecs)[0]\n",
    "        input_num_capsule = K.shape(u_vecs)[1]\n",
    "        u_hat_vecs = K.reshape(u_hat_vecs, (batch_size, input_num_capsule,\n",
    "                                            self.num_capsule, self.dim_capsule))\n",
    "        u_hat_vecs = K.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n",
    "        # final u_hat_vecs.shape = [None, num_capsule, input_num_capsule, dim_capsule]\n",
    "\n",
    "        b = K.zeros_like(u_hat_vecs[:, :, :, 0])  # shape = [None, num_capsule, input_num_capsule]\n",
    "        for i in range(self.routings):\n",
    "            b = K.permute_dimensions(b, (0, 2, 1))  # shape = [None, input_num_capsule, num_capsule]\n",
    "            c = K.softmax(b)\n",
    "            c = K.permute_dimensions(c, (0, 2, 1))\n",
    "            b = K.permute_dimensions(b, (0, 2, 1))\n",
    "            outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))\n",
    "            if i < self.routings - 1:\n",
    "                b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])\n",
    "\n",
    "        return outputs\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return (None, self.num_capsule, self.dim_capsule)\n",
    "    \n",
    "def squash(x, axis=-1):\n",
    "    # s_squared_norm is really small\n",
    "    # s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()\n",
    "    # scale = K.sqrt(s_squared_norm)/ (0.5 + s_squared_norm)\n",
    "    # return scale * x\n",
    "    s_squared_norm = K.sum(K.square(x), axis, keepdims=True)\n",
    "    scale = K.sqrt(s_squared_norm + K.epsilon())\n",
    "    return x / scale\n",
    "    \n",
    "# from keras.optimizers import *\n",
    "# from keras import initializers, regularizers, constraints, optimizers, layers\n",
    "class AdamW(Optimizer):\n",
    "    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, weight_decay=1e-4,  # decoupled weight decay (1/4)\n",
    "                 epsilon=1e-8, decay=0., **kwargs):\n",
    "        super(AdamW, self).__init__(**kwargs)\n",
    "        with K.name_scope(self.__class__.__name__):\n",
    "            self.iterations = K.variable(0, dtype='int64', name='iterations')\n",
    "            self.lr = K.variable(lr, name='lr')\n",
    "            self.beta_1 = K.variable(beta_1, name='beta_1')\n",
    "            self.beta_2 = K.variable(beta_2, name='beta_2')\n",
    "            self.decay = K.variable(decay, name='decay')\n",
    "            # decoupled weight decay (2/4)\n",
    "            self.wd = K.variable(weight_decay, name='weight_decay')\n",
    "        self.epsilon = epsilon\n",
    "        self.initial_decay = decay\n",
    "\n",
    "#     @interfaces.legacy_get_updates_support\n",
    "    def get_updates(self, loss, params):\n",
    "        grads = self.get_gradients(loss, params)\n",
    "        self.updates = [K.update_add(self.iterations, 1)]\n",
    "        wd = self.wd  # decoupled weight decay (3/4)\n",
    "\n",
    "        lr = self.lr\n",
    "        if self.initial_decay > 0:\n",
    "            lr *= (1. / (1. + self.decay * K.cast(self.iterations,\n",
    "                                                  K.dtype(self.decay))))\n",
    "\n",
    "        t = K.cast(self.iterations, K.floatx()) + 1\n",
    "        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /\n",
    "                     (1. - K.pow(self.beta_1, t)))\n",
    "\n",
    "        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n",
    "        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n",
    "        self.weights = [self.iterations] + ms + vs\n",
    "\n",
    "        for p, g, m, v in zip(params, grads, ms, vs):\n",
    "            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g\n",
    "            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)\n",
    "            # decoupled weight decay (4/4)\n",
    "            p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon) - lr * wd * p\n",
    "\n",
    "            self.updates.append(K.update(m, m_t))\n",
    "            self.updates.append(K.update(v, v_t))\n",
    "            new_p = p_t\n",
    "\n",
    "            # Apply constraints.\n",
    "            if getattr(p, 'constraint', None) is not None:\n",
    "                new_p = p.constraint(new_p)\n",
    "\n",
    "            self.updates.append(K.update(p, new_p))\n",
    "        return self.updates\n",
    "\n",
    "    def get_config(self):\n",
    "        config = {'lr': float(K.get_value(self.lr)),\n",
    "                  'beta_1': float(K.get_value(self.beta_1)),\n",
    "                  'beta_2': float(K.get_value(self.beta_2)),\n",
    "                  'decay': float(K.get_value(self.decay)),\n",
    "                  'weight_decay': float(K.get_value(self.wd)),\n",
    "                  'epsilon': self.epsilon}\n",
    "        base_config = super(AdamW, self).get_config()\n",
    "        return dict(list(base_config.items()) + list(config.items()))\n",
    "\n",
    "from __future__ import print_function\n",
    "from keras.engine.topology import Layer \n",
    "class Position_Embedding(Layer): \n",
    "    def __init__(self, size=None, mode='sum', **kwargs):        \n",
    "        self.size = size         \n",
    "        self.mode = mode       \n",
    "        super(Position_Embedding, self).__init__(**kwargs) \n",
    "\n",
    "    def call(self, x): \n",
    "        if (self.size == None) or (self.mode == 'sum'):            \n",
    "            self.size = int(x.shape[-1])        \n",
    "            batch_size, seq_len = K.shape(x)[0], K.shape(x)[1]        \n",
    "            position_j = 1. / K.pow(10000., \\\n",
    "                2 * K.arange(self.size / 2, dtype='float32') / self.size)        \n",
    "            position_j = K.expand_dims(position_j, 0)        \n",
    "            position_i = K.cumsum(K.ones_like(x[:, :, 0]), 1)-1     \n",
    "            position_i = K.expand_dims(position_i, 2)        \n",
    "            position_ij = K.dot(position_i, position_j)        \n",
    "            position_ij = K.concatenate([K.cos(position_ij), K.sin(position_ij)], 2) \n",
    "            if self.mode == 'sum': \n",
    "                return position_ij + x \n",
    "            elif self.mode == 'concat': \n",
    "                return K.concatenate([position_ij, x], 2) \n",
    "\n",
    "    def compute_output_shape(self, input_shape): \n",
    "        if self.mode == 'sum': \n",
    "            return input_shape \n",
    "        elif self.mode == 'concat': \n",
    "            return (input_shape[0], input_shape[1], input_shape[2]+self.size)\n",
    "\n",
    "from keras.layers import *\n",
    "def model_lstm_atten():\n",
    "    \n",
    "    inp_active = Input(shape=(maxlen,))\n",
    "    inp_usage = Input(shape=(maxlen,))\n",
    "#     aux = Input(shape=(len(feature_name),))\n",
    "    \n",
    "    # Stage Active\n",
    "    x = Embedding(a_max_features, embed_size, weights=[a_embedding_matrix], trainable=False)(inp_active)\n",
    "    x = Position_Embedding()(x)\n",
    "    x = SpatialDropout1D(0.1)(x)\n",
    "    x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\n",
    "    y = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\n",
    "    z = Conv1D(64, kernel_size = 1, kernel_initializer=initializers.he_uniform(seed=2019), activation = \"tanh\")(y)\n",
    "    \n",
    "    atten_1 = Attention(maxlen)(x) # skip connect\n",
    "    atten_2 = Attention(maxlen)(y)\n",
    "    avg_pool = GlobalAveragePooling1D()(y)\n",
    "    max_pool = GlobalMaxPooling1D()(y)\n",
    "    max_pool1 = GlobalMaxPooling1D()(z)\n",
    "    \n",
    "    convs = []\n",
    "    filter_sizes = [2, 3, 5, 8]\n",
    "    for fsz in filter_sizes:\n",
    "        l_conv = Conv1D(filters=maxlen, kernel_size=fsz, activation='relu')(y)\n",
    "        l_pool = MaxPooling1D(maxlen - fsz + 1)(l_conv)\n",
    "        l_pool = Flatten()(l_pool)\n",
    "        convs.append(l_pool)\n",
    "    text_cnn = concatenate(convs, axis=1)\n",
    "    \n",
    "    # Stage Usage\n",
    "    x = Embedding(u_max_features, embed_size, weights=[u_embedding_matrix], trainable=False)(inp_usage)\n",
    "    x = Position_Embedding()(x)\n",
    "    x = SpatialDropout1D(0.1)(x)\n",
    "    x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)\n",
    "    y = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\n",
    "    \n",
    "    atten_3 = Attention(maxlen)(x)\n",
    "    atten_4 = Attention(maxlen)(y)\n",
    "    \n",
    "    convs = []\n",
    "    filter_sizes = [2, 4, 6, 10]\n",
    "    for fsz in filter_sizes:\n",
    "        l_conv = Conv1D(filters=maxlen, kernel_size=fsz, activation='relu')(y)\n",
    "        l_pool = MaxPooling1D(maxlen - fsz + 1)(l_conv)\n",
    "        l_pool = Flatten()(l_pool)\n",
    "        convs.append(l_pool)\n",
    "    text_cnn_1 = concatenate(convs, axis=1)\n",
    "    \n",
    "#     x = Dense(32)(aux)\n",
    "#     x = BatchNormalization()(x)\n",
    "    \n",
    "    conc = concatenate([atten_1, atten_2, avg_pool, max_pool, max_pool1, text_cnn,\n",
    "                        atten_3, atten_4, text_cnn_1])\n",
    "    conc = Dense(256)(conc)\n",
    "    conc = BatchNormalization()(conc)\n",
    "    conc = PReLU()(conc)\n",
    "    conc = Dropout(0.2)(conc)\n",
    "    \n",
    "    conc = Dense(128)(conc)\n",
    "    conc = BatchNormalization()(conc)\n",
    "#     conc = PReLU()(conc)\n",
    "\n",
    "    outp = Dense(6, activation=\"softmax\")(conc)    \n",
    "\n",
    "    model = Model(inputs=[inp_active,inp_usage], outputs=outp)\n",
    "    \n",
    "    return model\n",
    "\n",
    "embed_size = 300 # how big is each word vector\n",
    "a_max_features = 9000 # how many unique words to use (i.e num rows in embedding vector)\n",
    "u_max_features = 30000\n",
    "maxlen = 200 # max number of words in a question to use\n",
    "\n",
    "train = pd.read_csv(\"../input/age_train.csv\",names=['uid','age_group']).sort_values(by=['uid'])\n",
    "test = pd.read_csv(\"../input/age_test.csv\",names=['uid']).sort_values(by=['uid'])\n",
    "aux = pd.read_pickle(\"../feature/all_data_v1.pickle\")\n",
    "\n",
    "from keras.utils import np_utils\n",
    "if os.path.exists(\"active_train_X.npy\"):\n",
    "    a_train_X = np.load(\"active_train_X.npy\")\n",
    "    a_test_X = np.load(\"active_test_X.npy\")\n",
    "    a_train_y = np.load(\"active_train_y.npy\")\n",
    "    a_word_index = dict(pd.read_pickle(\"active_word_index.pickle\").values)\n",
    "    \n",
    "else:\n",
    "    a_train_X, a_test_X, a_train_y, a_word_index = load_and_prec(\"active\")\n",
    "    a_train_y = np_utils.to_categorical(a_train_y-1)\n",
    "    np.save(\"active_train_y.npy\",a_train_y)\n",
    "    np.save(\"active_train_X.npy\",a_train_X)\n",
    "    np.save(\"active_test_X.npy\",a_test_X)\n",
    "    a_w2i = pd.DataFrame()\n",
    "    a_w2i['appid'] = a_word_index.keys()\n",
    "    a_w2i['index'] = a_word_index.values()\n",
    "    a_w2i.to_pickle(\"active_word_index.pickle\")\n",
    "\n",
    "if os.path.exists(\"usage_train_X.npy\"):\n",
    "    u_train_X = np.load(\"usage_train_X.npy\")\n",
    "    u_test_X = np.load(\"usage_test_X.npy\")\n",
    "    u_train_y = np.load(\"usage_train_y.npy\")\n",
    "    u_word_index = dict(pd.read_pickle(\"usage_word_index.pickle\").values)\n",
    "    \n",
    "else:\n",
    "    u_train_X, u_test_X, u_train_y, u_word_index = load_and_prec(\"usage\")\n",
    "    u_train_y = np_utils.to_categorical(u_train_y-1)\n",
    "    np.save(\"usage_train_y.npy\",u_train_y)\n",
    "    np.save(\"usage_train_X.npy\",u_train_X)\n",
    "    np.save(\"usage_test_X.npy\",u_test_X)\n",
    "    u_w2i = pd.DataFrame()\n",
    "    u_w2i['appid'] = u_word_index.keys()\n",
    "    u_w2i['index'] = u_word_index.values()\n",
    "    u_w2i.to_pickle(\"usage_word_index.pickle\")\n",
    "\n",
    "embedding_matrix = load_glove(a_word_index,a_max_features,'0')\n",
    "# embedding_matrix_2 = load_fasttext(a_word_index)\n",
    "# embedding_matrix_3 = load_w2v(a_word_index,a_max_features,'0')\n",
    "# a_embedding_matrix = np.concatenate([embedding_matrix_1,embedding_matrix_3],axis=1)\n",
    "\n",
    "embedding_matrix = load_glove(u_word_index,u_max_features,'1')\n",
    "# embedding_matrix_2 = load_fasttext(u_word_index)\n",
    "# embedding_matrix_3 = load_w2v(u_word_index,u_max_features,'1')\n",
    "# u_embedding_matrix = np.mean([embedding_matrix_1,embedding_matrix_3],axis=0)\n",
    "\n",
    "a_embedding_matrix.shape,u_embedding_matrix.shape\n",
    "\n",
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = \"0,2,3\"\n",
    "\n",
    "from keras.utils.training_utils import multi_gpu_model \n",
    "\n",
    "DATA_SPLIT_SEED = 2019\n",
    "cb = [EarlyStopping(monitor='val_loss', patience=3, verbose=0),\n",
    "      CyclicLR(base_lr=0.001, max_lr=0.002,\n",
    "               step_size=300., mode='exp_range',\n",
    "               gamma=0.99994)]\n",
    "\n",
    "train_meta = np.zeros(a_train_y.shape)\n",
    "test_meta = np.zeros((a_test_X.shape[0],6))\n",
    "splits = list(KFold(n_splits=5, shuffle=True, random_state=DATA_SPLIT_SEED).split(a_train_X, a_train_y))\n",
    "for idx, (train_idx, valid_idx) in enumerate(splits):\n",
    "    K.clear_session()\n",
    "    a_X_train, u_X_train = a_train_X[train_idx], u_train_X[train_idx]\n",
    "    a_X_val, u_X_val = a_train_X[valid_idx], u_train_X[valid_idx]\n",
    "    y_train = a_train_y[train_idx]\n",
    "    y_val = a_train_y[valid_idx]\n",
    "    \n",
    "#     Aux_train = aux_train[train_idx]\n",
    "#     Aux_val = aux_train[valid_idx]\n",
    "    \n",
    "    model = model_lstm_atten()\n",
    "    model = multi_gpu_model(model, gpus=8)\n",
    "    model.compile(loss='categorical_crossentropy', optimizer=RAdam(lr=0.001), metrics=['acc'])\n",
    "    print(\"MODEL COMPLIE...\")\n",
    "    lookahead = Lookahead(k=5, alpha=0.5) # Initialize Lookahead\n",
    "    lookahead.inject(model) # add into model\n",
    "    print(\"LOOKAHEAD INJECT...\")\n",
    "    model.summary()\n",
    "    model.fit([a_X_train,u_X_train], y_train, batch_size=1024, epochs=1, validation_data=([a_X_val,u_X_val], y_val), callbacks = cb, verbose=1)\n",
    "    pred_val_y = model.predict([a_X_val,u_X_val], batch_size=2048, verbose=1)\n",
    "    pred_test_y = model.predict([a_test_X,u_test_X], batch_size=2048, verbose=1)\n",
    "    train_meta[valid_idx] = pred_val_y\n",
    "    test_meta += pred_test_y / len(splits)\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "print(accuracy_score(np.argmax(u_train_y,axis=1),np.argmax(train_meta,axis=1)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
